{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a271a570",
   "metadata": {},
   "source": [
    "# Get resource from google scholar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1baf4c57",
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing the requests library \n",
    "import requests \n",
    "import os\n",
    "path = os.getcwd()\n",
    "keyword = 'obesity' \n",
    "# insert your auth key here, you need a proxies account and get free api calls\n",
    "auth_key = \"040d202dffa14fe75209dac7b2a872a1_sr98766_ooPq87\"\n",
    "URL = \"http://api.proxiesapi.com\"\n",
    "for i in range(0,110,10):\n",
    "    url = \"https://scholar.google.com/scholar?start={0}&q={1}&hl=en&as_sdt=8,5\".format(i,keyword)\n",
    "    # defining a params dict for the parameters to be sent to the API \n",
    "    PARAMS = {'auth_key':auth_key, 'url':url} \n",
    "    # sending get request and saving the response as response object \n",
    "    r = requests.get(url = URL, params = PARAMS)\n",
    "    #make soup to get title from google scholar html\n",
    "    soup = BeautifulSoup(r.content,'lxml')\n",
    "    #save the results to txt file and name it as {keyword}.txt \n",
    "    with open(path + \"/{0}.txt\".format(keyword), 'a+') as f:\n",
    "        #title part\n",
    "        for item in soup.select('[data-clk-atid]'):\n",
    "            # get rid of [pdf] items\n",
    "            if item.get_text().startswith('['):\n",
    "                continue\n",
    "            else:\n",
    "                f.write(item.get_text())\n",
    "                f.write('\\n')\n",
    "                print(item.get_text())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fcbe768",
   "metadata": {},
   "source": [
    "# Text preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c962f43d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 6: expected 1 fields, saw 3\\nSkipping line 28: expected 1 fields, saw 3\\nSkipping line 41: expected 1 fields, saw 2\\nSkipping line 71: expected 1 fields, saw 2\\nSkipping line 75: expected 1 fields, saw 2\\nSkipping line 83: expected 1 fields, saw 4\\nSkipping line 85: expected 1 fields, saw 3\\nSkipping line 93: expected 1 fields, saw 2\\nSkipping line 94: expected 1 fields, saw 3\\nSkipping line 103: expected 1 fields, saw 2\\n'\n",
      "b'Skipping line 67: expected 1 fields, saw 2\\nSkipping line 70: expected 1 fields, saw 3\\nSkipping line 75: expected 1 fields, saw 2\\nSkipping line 79: expected 1 fields, saw 2\\nSkipping line 93: expected 1 fields, saw 2\\nSkipping line 104: expected 1 fields, saw 3\\n'\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "obesity_data = pd.read_csv('obesity.txt',error_bad_lines=False,header=None)\n",
    "cancer_data = pd.read_csv('cancer.txt',error_bad_lines=False,header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "bbb477c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#add column name\n",
    "obesity_data.columns=['text']\n",
    "cancer_data.columns=['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "5dc4f652",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 100 entries, 0 to 99\n",
      "Data columns (total 1 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   text    100 non-null    object\n",
      "dtypes: object(1)\n",
      "memory usage: 928.0+ bytes\n"
     ]
    }
   ],
   "source": [
    "#show information about obesity_data dataframe\n",
    "obesity_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "9f30ef5f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 100 entries, 0 to 99\n",
      "Data columns (total 1 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   text    100 non-null    object\n",
      "dtypes: object(1)\n",
      "memory usage: 928.0+ bytes\n"
     ]
    }
   ],
   "source": [
    "#show information about cancer_data dataframe\n",
    "cancer_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2ca2288c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "#remove all the non aplpha characters\n",
    "def remove_not_alp(text):\n",
    "    return re.sub('[^a-zA-Z]', ' ', text)\n",
    "obesity_data['text'] = obesity_data['text'].apply(remove_not_alp)\n",
    "cancer_data['text'] = cancer_data['text'].apply(remove_not_alp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "8339453c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#upper to lower\n",
    "def convert_to_lower(text):\n",
    "    return text.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "5af6af2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "obesity_data['text'] = obesity_data['text'].apply(convert_to_lower)\n",
    "cancer_data['text'] = cancer_data['text'].apply(convert_to_lower)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4fb7e90",
   "metadata": {},
   "source": [
    "# Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "a7fd0d9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "93891dd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#convert sentences to tokenizations\n",
    "def sentence_to_tokenization(text):\n",
    "    return word_tokenize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "4937a177",
   "metadata": {},
   "outputs": [],
   "source": [
    "obesity_data['text'] = obesity_data['text'].apply(sentence_to_tokenization)\n",
    "cancer_data['text'] = cancer_data['text'].apply(sentence_to_tokenization)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "1ca8c2f4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[obesity, and, the, environment, where, do, we...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[obesity, as, a, medical, problem]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[medical, consequences, of, obesity]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[the, epidemiology, of, obesity]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[focus, on, obesity]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>[the, disease, burden, associated, with, overw...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>[apparatus, and, methods, for, treatment, of, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>[genetics, of, obesity]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>[the, genetics, of, obesity]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>[childhood, obesity]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text\n",
       "0   [obesity, and, the, environment, where, do, we...\n",
       "1                  [obesity, as, a, medical, problem]\n",
       "2                [medical, consequences, of, obesity]\n",
       "3                    [the, epidemiology, of, obesity]\n",
       "4                                [focus, on, obesity]\n",
       "..                                                ...\n",
       "95  [the, disease, burden, associated, with, overw...\n",
       "96  [apparatus, and, methods, for, treatment, of, ...\n",
       "97                            [genetics, of, obesity]\n",
       "98                       [the, genetics, of, obesity]\n",
       "99                               [childhood, obesity]\n",
       "\n",
       "[100 rows x 1 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obesity_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59cf08be",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
